print "Program Begins..."

# option1:
# inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/testRanking_20140514_K1_B_option1_OR_raw"
# option2:
# inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/testRanking_20140514_K1_B_option2_OR_raw"
# option3:
# inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/testRanking_20140514_K1_B_option3_OR_raw"
# option4:
# inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/testRanking_20140514_K1_B_option4_OR_raw"
# option5:
# inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/testRanking_20140514_K1_B_option5_OR_raw"
# option6:
# inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/testRanking_20140514_K1_B_option6_OR_raw"
# option7:
inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/testRanking_20140514_K1_B_option4_AND_TOP2M_raw"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split("\t")
    if len(lineElements) == 6 and lineElements[5] == "WeiIRTK":
        print line.strip()
    else:
        pass
inputFileHandler.close()
print "OVERALL:"
print "inputFileName:",inputFileName
exit(1)


'''
existingQueryTermDict = {}
inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/ALL_queryTermThresholdsKeptBasedOnPercentage20130508.txt"
inputFileHandler = open(inputFileName1,"r")
currentLine = inputFileHandler.readline()

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTerm = lineElements[0]
    if currentTerm not in existingQueryTermDict:
        existingQueryTermDict[currentTerm] = 1
print "len(existingQueryTermDict): ",len(existingQueryTermDict)
inputFileHandler.close()

humanQueryTermDict = {}
inputFileName2 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/2014JanPruningResults/reproduceRomanQueryEffectiveness_20140121Night/04-06.topics.701-850.polyIRTKCompatibleMode"
inputFileHandler = open(inputFileName2,"r")
for line in inputFileHandler.readlines():
    data = line.strip().split(":")[1]
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    queryTermList = data.strip().split(" ")
    for queryTerm in queryTermList:
        if queryTerm not in humanQueryTermDict and queryTerm.strip() != "":
            humanQueryTermDict[queryTerm.lower()] = 1
print "len(humanQueryTermDict): ",len(humanQueryTermDict)

print "**********"
for term in humanQueryTermDict:
    if term not in existingQueryTermDict:
        print term
print "**********"
exit(1)
'''


# step1
# script for transferring the format
outputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/2014JanPruningResults/reproduceRomanQueryEffectiveness_20140121Night/OR_Semantics/TCP-QV/OR_top1000_50%_tb04-06_final"
# outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/pruningProjectResults/reproduceRomanQueryEffectiveness_20140121Night/AND_Semantics/AND_top10000_1%_tb04-06_final"
outputFileHandler = open(outputFileName,"w")

inputFileNameList = []
inputFileName1 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/2014JanPruningResults/reproduceRomanQueryEffectiveness_20140121Night/OR_Semantics/TCP-QV/OR_top1000_50%_tb04-06_raw"

# inputFileName1 = "/data3/obukai/workspace/web-search-engine-wei-2014-Jan/polyIRIndexer/2014JanPruningResults/ORSemantics/relRank20140121Morning_relRank_stepFactor_point5/humanJudgeQueries/rawResults_50%_TOP1000_OR_trec_format_20140123Afternoon_PART_OF"
# inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/pruningProjectResults/reproduceRomanQueryEffectiveness_20140121Night/AND_Semantics/AND_top10000_1%_tb04-06_raw"
# inputFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/pruningProjectResults/reproduceRomanQueryEffectiveness_20140121Night/AND_Semantics/AND_top10000_10%_tb04-06_final_Part2"
inputFileNameList.append(inputFileName1)
# inputFileNameList.append(inputFileName2)
for inputFileName in inputFileNameList: 
    inputFileHandler = open(inputFileName,"r")
    for line in inputFileHandler.readlines():
        if line.strip().endswith("WeiIRTK"):
            outputFileHandler.write(line)
    inputFileHandler.close()

outputFileHandler.close()
print "Overall:"
print "inputFileName1: ",inputFileName1
# print "inputFileName2: ",inputFileName2
print "outputFileName: ",outputFileName
exit(1)


# want to compare these two files, why there are differences?
qidsWithNumOfResultsDict1 = {}
qidsWithNumOfResultsDict2 = {}
currentQID = ""
currentQIDNumOfResults = 0
inputFileName1 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/2014JanPruningResults/ANDSemantics/relRank20140113Night/rawResults_1%_TOP10_AND_20140113Night"
inputFileName2 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/2014JanPruningResults/ANDSemantics/relRank20140113Night/rawResults_1%_TOP10_AND_20140119Night_FOR_20140113Night"
inputFileHandler1 = open(inputFileName1,"r")
for line in inputFileHandler1.readlines():
    if line.strip().startswith("qid:"):
        currentQID = line.strip().split(" ")[1]
    if line.strip().startswith("Showing"):
        currentQIDNumOfResults = int( line.strip().strip().split(" ")[1] )
        if currentQID not in qidsWithNumOfResultsDict1:
            qidsWithNumOfResultsDict1[currentQID] = currentQIDNumOfResults
print "len(qidsWithNumOfResultsDict1): ",len(qidsWithNumOfResultsDict1)
print "qidsWithNumOfResultsDict1['95005']: ",qidsWithNumOfResultsDict1['95005']
inputFileHandler1.close()

inputFileHandler2 = open(inputFileName2,"r")
for line in inputFileHandler2.readlines():
    if line.strip().startswith("qid:"):
        currentQID = line.strip().split(" ")[1]
    if line.strip().startswith("Showing"):
        currentQIDNumOfResults = int( line.strip().strip().split(" ")[1] )
        if currentQID not in qidsWithNumOfResultsDict1:
            qidsWithNumOfResultsDict1[currentQID] = currentQIDNumOfResults    

exit(1)

inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version3_20140119Afternoon/relRank_20140119Night_Probability_toolkit_friendly"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    key = lineElements[0]
    probability = float(lineElements[1])
    if probability == 0.0:
        print key,probability
inputFileHandler.close()
exit(1)




outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version3_20140119Afternoon/predefinedClassLabelUpperBoundOfRelRank20140119Afternoon"
outputFileHandler = open(outputFileName,"w")

inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/predefinedClassLabelLowerBoundBasedOnListLength_numOfPieces_stepGap_one3rd_20140118Night"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    classLabel = lineElements[0]
    numOfPieces = lineElements[2]
    outputFileHandler.write(classLabel + " " + numOfPieces + "\n")
inputFileHandler.close()
outputFileHandler.close()
print "Overall:"
print "inputFileName: ",inputFileName
print "outputFileName: ",outputFileName
exit(1)


outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version2_20140112Night/relRank_20140119Night_Probability_toolkit_friendly"
outputFileHandler = open(outputFileName,"w")

inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/relRank_20140119Afternoon_Probability.csv"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    classLabelBasedOnListLengthPart1 = lineElements[0]
    for indexPart2,probablity in enumerate( lineElements[1:] ):
        key = classLabelBasedOnListLengthPart1 + "_" + str(indexPart2)
        outputFileHandler.write(key + " " + str(probablity) + "\n")
inputFileHandler.close()
outputFileHandler.close()
print "Overall:"
print "inputFileName: ",inputFileName
print "outputFileName: ",outputFileName
exit(1)

initValue = 1.0
previousUpperBound = initValue
print previousUpperBound,
for i in range(1,34):
    currentUpperBound = previousUpperBound - previousUpperBound / 3
    print currentUpperBound,
    previousUpperBound = currentUpperBound
exit(1)

cellValueDict1 = {}
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/debug_3"
inputFileHandler = open(inputFileName,"r")
tempCounter = 0
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    if len(lineElements) == 3:
        id1 = lineElements[0]
        id2 = lineElements[1]
        key = id1 + "_" + id2
        currentValue = int(lineElements[2])
        tempCounter += currentValue
        cellValueDict1[key] = currentValue
print "tempCounter: ",tempCounter
print "len(cellValueDict1): ",len(cellValueDict1)
inputFileHandler.close()

cellValueDict2 = {}
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/debug_2"
inputFileHandler = open(inputFileName,"r")
tempCounter = 0
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    if len(lineElements) == 3:
        id1 = lineElements[0]
        id2 = lineElements[1]
        key = id1 + "_" + id2
        currentValue = int(lineElements[2])
        tempCounter += currentValue
        cellValueDict2[key] = currentValue
        if key in cellValueDict1:
            if currentValue == cellValueDict1[key]:
                pass
            else:
                print "critical error"
                print "id1: ",id1
                print "id2: ",id2
                print "key: ",key
                print "lineElements[2]: ",lineElements[2]
                print "cellValueDict1[key]: ",cellValueDict1[key]
                exit(1)
        else:
            pass
print "tempCounter: ",tempCounter
print "len(cellValueDict2): ",len(cellValueDict2)
inputFileHandler.close()

for key in cellValueDict1:
    if cellValueDict1[key] == cellValueDict2[key]:
        pass
    else:
        print "critical error"
        exit(1)
exit(1)







classLabelWithNumOfPiecesDict = {}
inputAuxFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/predefinedClassLabelLowerBoundBasedOnListLength_numOfPieces_stepGap_one3rd_20140118Night"
inputFileHanlder = open(inputAuxFileName,"r")
for line in inputFileHanlder.readlines():
    lineElements = line.strip().split(" ")
    classLabel = lineElements[0]
    numOfPiecesDecided = int(lineElements[2])
    if classLabel not in classLabelWithNumOfPiecesDict:
         classLabelWithNumOfPiecesDict[classLabel] = numOfPiecesDecided
inputFileHanlder.close()
print "classLabelWithNumOfPiecesDict['26']: ",classLabelWithNumOfPiecesDict['26']
    
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/debug_2"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    print lineElements[0],len(lineElements)-1
    if classLabelWithNumOfPiecesDict[lineElements[0]] == len(lineElements)-1:
        pass
    else:
        print "critical error"
        exit(1)

inputFileHandler.close()
exit(1)

# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/termPieceInfoForQueryTerms_stepGap_one3rd"
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/termPieceInfoForRandomPostingTerms_stepGap_one3rd"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    numToCheck = 4 + int(lineElements[3]) * 2
    if len(lineElements) == numToCheck:
        pass
    else:
        print "error:"
        print "len(lineElements): ",len(lineElements)
        print "numToCheck: ",numToCheck
        print line.strip()
        exit(1)
print "Passed."
inputFileHandler.close()
exit(1)

inputFileName = "/Users/weijiang/Desktop/dataWorkOn20140111Night/relRank_20140111Night_Numerator.csv_OLD"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/Users/weijiang/Desktop/dataWorkOn20140111Night/relRank_20140111Night_Numerator.csv"
outputFileHandler = open(outputFileName,"w")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(",")
    numOfRangesThatIsMeaningful = int( lineElements[3] )
    print "len(lineElements): ",len(lineElements)
    print "numOfRangesThatIsMeaningful: ",numOfRangesThatIsMeaningful
    outputLine = lineElements[0] + " " + lineElements[1] + " " + lineElements[2] + " " + lineElements[3] + " " + lineElements[4] + " "
    lowerBound = 5
    upperBound = 5 + numOfRangesThatIsMeaningful
    for currentValue in lineElements[lowerBound:upperBound]:
        # print i
        outputLine += currentValue + " "
    outputLine = outputLine.strip() + "\n"
    outputFileHandler.write(outputLine)
inputFileHandler.close()
outputFileHandler.close()

print "Overall Processing Statistics:"
print "inputFileName: ",inputFileName
print "outputFileName: ",outputFileName
print "Program Ends."
